#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# 还原短url，将其保存到RegularUrl.txt
# 从还原后的url中抽取host字段，保存到host_from_short.txt

from urllib import request,error
from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool
import datetime
from urllib.parse import urlparse #python3 only

# def GetUrl(str):
#     try:
#         fp = request.urlopen(str, timeout = 5)
#         url = fp.geturl()
#         return url
#     except:
#         return ''

# try:
#     shorturl = open("E:\Hosts\\xuanwulab\\twitter\ShortUrl.txt","r").readlines()
#
#     pool = ThreadPool(8)  # 不设置的话，默认当前计算机的内核数
#
#     url_list = list(pool.map(GetUrl,shorturl)) #python3中map函数若要得到列表，需使用list()转换
#     pool.close()
#     pool.join()
#
#     file_url = open("E:\Hosts\\xuanwulab\\twitter\\url_from_short.txt", "w")
#     for u in url_list :
#         #从转换完成的正常url中，获取host
#         hostname = urlparse(u).netloc
#         if hostname == "github.com":
#             file_url.write(u + '\n')
#     file_url.close()
#
# except Exception as e:
#     print(e)

#或者直接从已得到的正常url中获取host=github.com的url
try:
    url_list = open("E:\Hosts\\xuanwulab\RegularUrl.txt","r").readlines()
    file_url = open("E:\Hosts\\xuanwulab\\twitter\\url_from_short.txt", "w")
    for u in url_list :
        #从转换完成的正常url中，获取host
        hostname = urlparse(u).netloc
        if hostname == "twitter.com":
            file_url.write(u)
    file_url.close()

except Exception as e:
    print(e)

